import Core.WindFunctions as Wind
import Core.MongoDB as MongoDB
import Core.Gadget as Gadget
import Core.IO as IO
import datetime as datetime
import pymongo
import random
import numpy as np
import DataProcess

from Core.Config import Config
config = Config()
database = config.DataBase()

#---Get Daily Bars---
def Download_DailyBar(database, instruments, datetime1, datetime2):
    #
    datetime2 = Gadget.ToUTCDateTime(datetime2)
    datetime2 = Gadget.ToLocalDateTime(datetime2)
    #
    if datetime1 != None:
        datetime1 = Gadget.ToUTCDateTime(datetime1)
        datetime1 = Gadget.ToLocalDateTime(datetime1)

    #
    sort = [("StdDateTime", pymongo.ASCENDING)]
    #
    db = database.client["Quote"]
    names = db.collection_names(False)
    #
    instruments = database.findWithFilter("Instruments", "Stock")
    instruments2 = database.findWithFilter("Instruments", "Index")
    instruments = instruments.extend(instruments2)

    i = 0
    for instrument in instruments:
        i = i + 1
        symbol = instrument["Symbol"]
        collName = symbol + "_Time_86400_Bar"

        #--暂时不处理摘牌股票---
        delistDate = None
        if "Delist_Date" in instrument:
            delistDate = Gadget.ToLocalDateTime(instrument["Delist_Date"])
            continue

        #--
        if datetime1 != None:
            beginDateTime = datetime1
        else:
            dataSeries = database.findWithFilter("Quote", collName, sort=sort)
            #---因为已经刨除了摘牌股票，所以在此大概率是新股票---
            count = len(dataSeries)
            if count == 0:
                beginDateTime = instrument["IPO_Date"]
                beginDateTime = Gadget.ToLocalDateTime(beginDateTime)
            else:
                lastItem = dataSeries[count - 1]
                lastDateTime = lastItem["StdDateTime"]
                print(symbol + " " + str(lastDateTime))
                beginDateTime = lastDateTime + datetime.timedelta(days=1)
                beginDateTime = Gadget.ToLocalDateTime(beginDateTime)

        endDateTime = datetime2
        print("Request " + symbol + " Begin " + str(beginDateTime) + " End " + str(endDateTime))
        #Wind.Request_Write_DailyBars(database, symbol, beginDateTime, endDateTime)
        kkwood = 0


#---Get Minute Bar---
def Download_MinutBar(database, datetime1, datetime2, instrumentList = None):

    #(2016,1,1,9,0,0) ~ (2016,6,8,16,0,0)
    #begDateTime = datetime(2016,7,7,9,0,0)
    #endDateTime = datetime(2016,7,7,16,0,0)
    begDateTime = datetime1
    begDateTime.hour = 9

    endDateTime = datetime2
    endDateTime.hour = 16

    if instrumentList == None:
        instrumentList = database.findWithFilter("Instruments","Stock",{})

    #Request_MinuteBars("000004.SZ",begDateTime,endDateTime)
    start = False
    for instrument in instrumentList:
        symbol = instrument["Symbol"]
        kkwood = 1
        #if symbol == "002778.SZ":
        #    start = True
        #if start == False:
        #    continue
        #HS300 --- 603993 --- 18600
        Wind.Request_Write_MinuteBars(database,symbol,begDateTime,endDateTime)


    '''
    #Check 60BarSeries
    for instrument in instruments:
        symbol = instrument[0]
        barSeries = database.getDataSeries(symbol + "_Time_60_Bar")
        #barSeries = database.getDataSeries("603993.SH_Time_60_Bar")
        count = 0
        for bar in barSeries:
            if math.isnan(bar["Close"]):
                datetime0 = bar["DateTime"]
                count = count + 1
        print(symbol + " #Total " + str(barSeries.count()) + " #Invalid " + str(count))
    '''


# ---从wind WSD 读取一些财务信息，然后变成因子存入---
def Download_Indicator(database, datetime1):
    # instrument = instruments["000001.SZ"]
    # fields = ["roe","roe_basic"]
    # reportDate = datetime(2015,12,31,0,0,0)
    # data = w.wss("000001.SZ", fields,"rptDate=20151231;rptType=1")
    # sReportDate = reportDate.strftime('%Y-%m-%d %H:%M:%S') + ".000"
    # target = {"ReportDate": sReportDate}
    # update = {"ROE_Basic":data.Data[0][0]}
    # database.upsert("Fundamental","000001.SZ_Fundamental", target, update)

    #
    instruments = database.findWithFilter("Instruments", "Stock", {})

    # ---遍历所有Instrument（Stock）---
    for instrument in instruments:
        # ---准备DataSeries容器---
        symbol = instrument.symbol
        indicator = "EstInstnum"
        size = 86400
        params = "CurrentYear"
        name = symbol + "_" + indicator + "_" + str(size) + "_" + params + "_FundamentalFactorValue"
        dataObjects = DataSeries(name)  # DataSeries

        # ---从Wind获取数据
        data = w.wsd(instrument.symbol, "est_instnum", "2013-01-01", "2013-12-31", "year=2013")
        # ---把从wind获取的数据填充进DataSeries容器
        i = 0
        for d in data.Data[0]:
            dataObject = {}
            sDateTime = Gadget.__ToDateTimeString__(data.Times[i])
            dataObject["DateTime"] = sDateTime
            dataObject["StdDateTime"] = data.Times[i]
            dataObject["Symbol"] = symbol
            dataObject["Indicator"] = indicator
            dataObject["Size"] = size
            dataObject["Value"] = d
            dataObject["Params"] = params
            dataObject["DataSeries"] = name
            dataObjects.add(dataObject)
            i = i + 1
        # ---把DataSeries储存到数据库---
        database.saveDataSeries(dataObjects)


def Download_HistoricalIndustry(database, datetime1, datetiem2):
    print("Downloading... HistoricalIndustry ")
    DataProcess.Download_Data_ByDate()


def Check():
    count = 0
    db = database.client["Quote"]
    coll =  db.collection_names(False)
    for name in coll:
        if "_Time_60_Bar" in name:
            count = count + 1
            symbol = name[:9]
            if symbol not in instrumentList:
                print(symbol)
            kkwood = 1


def CheckNoDailyQuoteNewStock(database, endDateTime):
    db = database.client["Quote"]
    names = db.collection_names(False)
    instruments = database.findWithFilter("Instruments", "Stock")

    #---
    noQuotes = []
    for instrument in instruments:
        symbol = instrument["Symbol"]
        name = symbol + "_Time_86400_Bar"
        existed = True
        if name not in names:
            noQuotes.append((symbol, instrument["IPO_Date"]))
            beginDateTime = Gadget.ToLocalDateTime(instrument["IPO_Date"])
            #
            print("Fix DailyBar: " + symbol + " IPODate:" + str(beginDateTime))
            Wind.Request_Write_DailyBars(database, symbol, beginDateTime, endDateTime)
            database.creatIndex("Quote", name,"Key")
            database.creatIndex("Quote", name, "StdDateTime")
            kkwood = 0
    #---
    print("Finish")


#---检查是否未建立Collection 或者Collection为空---
def CheckQuoteDailyBar(database, instrumentType, refDaterTime):
    #
    refDaterTime = Gadget.ToUTCDateTime(refDaterTime)
    #
    datetime1Field = "DateTime1"
    datetime2Field = "DateTime2"
    databaseName = instrumentType

    if instrumentType == "Stock":
        datetime1Field = "IPO_Date"
        datetime2Field = "Delist_Date"
        databaseName = "Quote"

    #
    db = database.client[databaseName]
    names = db.collection_names(False)

    noQuotes = []
    instruments = database.findWithFilter("Instruments", instrumentType)
    #instruments2 = database.findWithFilter("Instruments", "Index")
    #instruments = instruments.extend(instruments2)

    for instrument in instruments:
        symbol = instrument["Symbol"]
        name = symbol + "_Time_86400_Bar"
        datetime1 = instrument[datetime1Field]
        datetime2 = None
        if datetime2Field in instrument:
            datetime2 = instrument[datetime2Field]
            #delistDate = Gadget.ToLocalDateTime(instrument["Delist_Date"])
            #sDelistDate = str(delistDate)
            # print("DeList " + symbol + " DeList " + deList)

        if datetime1 < Gadget.ToUTCDateTime(datetime.datetime(1900,1,1)):
            print("NotSetupYet " + symbol)
            continue

        if datetime1 >= refDaterTime:
            print("NotSetupYet " + symbol)
            continue

        #---Not exist Collecction---
        if name not in names:
            #noQuotes.append((symbol,instrument.get("IPO_Date")))
            investType = instrument["InvestType1"]
            print("NoCollection " + symbol + " " + investType)
            continue

        #---Not exist Quote Data---
        stats = database.Stats(databaseName,name)
        if stats["count"] == 0:
            print("NoQuote " + symbol)
            kkwood = 0

    #noQuotes.sort(key=lambda x: x[1])
    #print(noQuotes)


#---检查是否丢失某天的Bar---
def CheckMissingDailyBars(database):

    instruments = database.findWithFilter("Instruments", "Stock")
    instruments2 = database.findWithFilter("Instruments", "Index")
    instruments = instruments.extend(instruments2)

    # datetime1 = Gadget.ToUTCDateTime(datetime1)
    # datetime2 = Gadget.ToUTCDateTime(datetime2)
    sort = [("StdDateTime", pymongo.ASCENDING)]

    # ---
    db = database.client["Quote"]
    names = db.collection_names(False)
    count = len(names)
    k = 0
    for instrument in instruments:
        symbol = instrument["Symbol"]
        name = symbol + "_Time_86400_Bar"
        k = k + 1
        print(name + " " + str(k))
        #if k < 109:
        #    continue

        dataSeries = database.findWithFilter("Quote", name, sort=sort)
        countDataSeries = len(dataSeries)
        if countDataSeries == 0:
            print("NoData: " + name)
            continue

        # ---Rebuild---
        datetime1 = dataSeries[0]["StdDateTime"]
        datetime2 = dataSeries[len(dataSeries) - 1]["StdDateTime"]
        filter = {}
        filter["StdDateTime"] = {"$gte": datetime1, "$lte": datetime2}
        bmDataSeries = database.findWithFilter("Quote", "000001.SH_Time_86400_Bar", filter, sort=sort)

        countBMDataSeries = len(bmDataSeries)

        if countBMDataSeries != countDataSeries:
            print("MissingDailyBar: " + name + " diff " + str(countBMDataSeries - countDataSeries) + " Count " + str(k))
        #continue

        if countBMDataSeries == countDataSeries:
            continue

        # ---Based on BM DataSeries---
        missingData = []
        for i in range(len(bmDataSeries)):
            stdDateTime = bmDataSeries[i]["StdDateTime"]
            element = Gadget.Find(dataSeries, stdDateTime, "Exact")
            if element == None:
                missingData.append(stdDateTime)
            kkwood = 0

        if len(missingData) > 0:
            missingDataBeginDateTime = missingData[0]
            missingDataEndDateTime = missingData[len(missingData) - 1]
            #Wind.Request_Write_DailyBars(database, symbol, missingDataBeginDateTime, missingDataEndDateTime)
            IO.WriteListToFile("C:/Output/" + name + ".txt",missingData)

    kkwood = 0


def TestPerformance(iteration=1000, isCombinedBar=True):
    #
    instruments = database.Find("Instruments", "Stock")

    #
    nums = []
    for i in range(iteration):
        #
        rad1 = random.randint(0, len(instruments)-1)
        instrument = instruments[rad1]

        #
        symbol = instrument["Symbol"]
        filter = {}
        databaseName = ""
        collectionName = ""
        if isCombinedBar:
            filter = {"Symbol": symbol}
            databaseName = "Stock"
            collectionName = "DailyBar"
        else:
            databaseName = "Quote"
            collectionName = symbol + "_Time_86400_Bar"

        datetime1 = datetime.datetime.now()
        # bars = database.Find(databaseName, collectionName, filter)
        dataSeries = database.GetDataSeries(symbol, dataType="DailyBar", instrumentType="Stock")
        datetime2 = datetime.datetime.now()
        # print(datetime1)
        # print(datetime2)
        diff = datetime2-datetime1
        print(symbol,diff.total_seconds())
        m = diff.seconds
        a = diff.microseconds
        nums.append(a * 0.001)
        pass

    print("Avrage time ", str(np.mean(nums)), "milliSec")


def CheckDiffTowDataBase():
    #
    instruments = database.Find("Instruments", "Stock")

    #
    count = 0
    for instrument in instruments:
        count += 1
        #
        symbol = instrument["Symbol"]
        print(symbol, count)

        #
        filter = {"Symbol": symbol}
        bars1 = database.Find("Stock", "DailyBar", filter)

        #
        bars2 = database.Find("Quote", symbol + "_Time_86400_Bar")

        #
        if len(bars1) != len(bars2):
            print("Error", symbol, len(bars1), len(bars2))


def CheckMissingIndustry(industryCategory="CS"):
    #
    instruments = database.find("Instruments", "Stock")
    countMissing = 0
    # 构建行业的列表（有多少行业）
    industries = []
    instrumentListByIndustry = {}

    for instrument in instruments:
        needToUpdate = False
        industry = instrument[industryCategory + "Industry1"]
        if industry == None:
            print(instrument["Symbol"] + " Null info" + " IPO " + Gadget.ToDateString(instrument["IPO_Date"]))
            needToUpdate = True
        #
        if needToUpdate:
            print(instrument["Symbol"], instrument["Description"])
            countMissing += 1

    #
    print("#Missing", countMissing)


def FixMissingIndustry(database, industryCategory="CS"):
    #
    instruments = database.find("Instruments", "Stock")
    countMissing = 0
    # 构建行业的列表（有多少行业）
    industries = []
    instrumentListByIndustry = {}

    missingSymbols = []
    for instrument in instruments:
        symbol = instrument["Symbol"]
        needToUpdate = False
        industry = instrument[industryCategory + "Industry1"]
        if industry == None:
            print(instrument["Symbol"] + " Null info" + " IPO " + Gadget.ToDateString(instrument["IPO_Date"]))
            needToUpdate = True
        #
        if needToUpdate:
            missingSymbols.append(symbol)

    #
    sDatetime = datetime.datetime.now()
    params = {"tradeDate": sDatetime}
    if industryCategory == "CS":
        params["industryType"] = 1
    elif industryCategory == "SW":
        params["industryType"] = 2
    #
    params = {"industryType": 1, "industryStandard": 1, "tradeDate": sDatetime}
    fields = "industry2"
    #
    params["industryStandard"] = 1
    industries1 = Wind.WSS(missingSymbols, fields, params)
    params["industryStandard"] = 2
    industries2 = Wind.WSS(missingSymbols, fields, params)
    params["industryStandard"] = 3
    industries3 = Wind.WSS(missingSymbols, fields, params)

    #
    for i in range(len(missingSymbols)):
        symbol = missingSymbols[i]
        dataObject = {}
        dataObject[industryCategory + "Industry1"] = industries1[i]["industry2"]
        dataObject[industryCategory + "Industry2"] = industries2[i]["industry2"]
        dataObject[industryCategory + "Industry3"] = industries3[i]["industry2"]
        if industryCategory == "CS":
            dataObject["Industry"] = industries1[i]["industry2"]
        #
        database.Upsert("Instruments", "Stock", {"Symbol": symbol}, dataObject)


Wind.w.start()
FixMissingIndustry(database, "CS")
CheckMissingIndustry("CS")

#--------------Code--------------
#---Connect to DataBase, Find Series 连接数据库---
# database = MongoDB.MongoDB("localhost","27017")

# ---Check---
#CheckQuoteDailyBar(database)
#CheckMissingDailyBars(database)

endDateTime = datetime.datetime(2017,9,16)
#CheckNoDailyQuoteNewStock(database, endDateTime)
#CheckQuoteDailyBar(database, "MutualFund", endDateTime)


TestPerformance(100, True)
k = 0

CheckDiffTowDataBase()
k = 1

# 启动Wind API
Wind.w.start()
begDateTime = datetime.datetime(2008,1,1)
endDateTime = datetime.datetime(2017,5,1)




# ---更新Assets列表---
#datetime0 = datetime.datetime(2005,1,1)
datetime0 = datetime.datetime(2017,9,15)
reportDate = datetime.datetime(2017,12,31)
#DataProcess.Request_Write_Stock_List(database, datetime0)
#DataProcess.Request_Write_MutualFund_List(database, datetime0, True)
#DataProcess.Request_Write_Index_List(database, datetime0)
#DataProcess.Request_Write_Future_List(database, datetime0)
#DataProcess.Request_Write_Future_List_Extension(database, datetime0)
kkwood = 1
#Wind.Request_Write_Stock_List_Extension(database,["mkt"],["Market"] )
kkwood = 0


# ---更新指数成分股---
#datetime1 = datetime.datetime(2016,8,1)
#datetime2 = datetime.datetime(2017,7,10)
#DataProcess.Request_Write_InstrumentList(database, endDateTime, "000016.SH")
kkwood = 0


# ---下载Bar数据---
#data = Wind.WSD_DailyQuote("000001.OF", begDateTime, endDateTime, "MutualFund")
#DataProcess.Download_DailyBars(database, ["000001.SZ","000002.SZ"], begDateTime, endDateTime)

# DataProcess.Automatic_Download_DailyQuote(database,endDateTime,instrumentType = "Stock")
# DataProcess.Automatic_Download_DailyQuote(database,endDateTime,instrumentType = "Index")
# DataProcess.Automatic_Download_DailyQuote(database,endDateTime,instrumentType = "Future")
# DataProcess.Automatic_Download_DailyQuote(database,endDateTime,instrumentType = "Bond")
# DataProcess.Automatic_Download_DailyQuote(database,endDateTime,instrumentType = "MutualFund")
kkwood = 2



